﻿using System;
using System.Data;
using System.Configuration;
using System.Linq;
using System.Web;
using System.Web.Security;
using System.Web.UI;
using System.Web.UI.HtmlControls;
using System.Web.UI.WebControls;
using System.Web.UI.WebControls.WebParts;
using System.Xml.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Data.SqlClient;
using System.IO;
using System.Net;
using System.Collections.Generic;

namespace ClubWebsite
{
    public class ScrapeFunctions
    {
        public enum ScrapeAction
        {
            RemoveAllTextBeforeARegexMatch = 1,
            RemoveAllTextAfterARegexMatch = 2,
            GetMatchesFromRegex = 3,
            ReplaceTextByRegex = 4,
            UpdateMatches = 5,
            InsertParentValueIntoText = 6,
            AddValueToNumber = 7,
            GetHTMLFromURL = 8,
            CleanHTML = 9
        }

        public static string ExecuteScrape(Guid scrapeID, string input)
        {
            // Get the steps to execute
            foreach (ScrapeStep scrapeStep in Functions.GetDatabaseDataContext()
                .ScrapeSteps
                .Where(ss => ss.ScrapeID == scrapeID)
                .OrderBy(ss => ss.ExecuteOrder))
            {
                // Check values have been given for the parameters required
                if ((scrapeStep.ScrapeStepAction.Parameter1Required != 0) && ((scrapeStep.Parameter1 == null) || (scrapeStep.Parameter1.Length == 0)))
                {
                    throw new ApplicationException("Parameter 1 is required.");
                }
                if ((scrapeStep.ScrapeStepAction.Parameter2Required != 0) && ((scrapeStep.Parameter2 == null) || (scrapeStep.Parameter2.Length == 0)))
                {
                    throw new ApplicationException("Parameter 2 is required.");
                }

                // Peform the action given
                switch ((ScrapeAction)Enum.ToObject(typeof(ScrapeAction), scrapeStep.Action))
                {
                    case ScrapeAction.GetHTMLFromURL:
                        if (scrapeStep.Parameter2 != null)
                        {
                            input = GetHtmlFromURL(scrapeStep.Parameter1, scrapeStep.Parameter2);
                        }
                        else
                        {
                            input = GetHtmlFromURL(scrapeStep.Parameter1, null);
                        }
                        break;

                    case ScrapeAction.CleanHTML:
                        input = CleanHTMLForRegex(input);
                        break;

                    /*case ScrapeAction.GetMatchesFromRegex:
                        MatchCollection matches = Regex.Matches(input, scrapeStep.Parameter1, RegexOptions.IgnoreCase | RegexOptions.Singleline);
                        break;*/

                    case ScrapeAction.ReplaceTextByRegex:
                        input = Regex.Replace(input, scrapeStep.Parameter1, scrapeStep.Parameter2, RegexOptions.IgnoreCase | RegexOptions.Singleline);
                        break;

                    case ScrapeAction.UpdateMatches:
                        List<DBMatch> matches = new List<DBMatch>();
                        foreach (System.Text.RegularExpressions.Match regexMatch in Regex.Matches(input, scrapeStep.Parameter1, RegexOptions.IgnoreCase | RegexOptions.Singleline))
                        {
                            DBMatch match = new DBMatch();
                            if ((regexMatch.Groups["DateTime"].Success) && (regexMatch.Groups["DateTime"].Value.Trim().Length > 0))
                            {
                                match.UTC = DateTime.Parse(regexMatch.Groups["DateTime"].Value.Trim()).ToUniversalTime();
                            }
                            else if (((regexMatch.Groups["Date"].Success) && (regexMatch.Groups["Date"].Value.Trim().Length > 0))
                                && ((regexMatch.Groups["Time"].Success) && (regexMatch.Groups["Time"].Value.Trim().Length > 0)))
                            {
                                match.UTC = DateTime.Parse(regexMatch.Groups["Date"].Value.Trim() + " " + regexMatch.Groups["Time"].Value.Trim()).ToUniversalTime();
                            }
                            if ((regexMatch.Groups["Round"].Success) && (regexMatch.Groups["Round"].Value.Trim().Length > 0))
                            {
                                match.Round = byte.Parse(regexMatch.Groups["Round"].Value.Trim());
                            }
                            if ((regexMatch.Groups["HomeTeam"].Success) && (regexMatch.Groups["HomeTeam"].Value.Trim().Length > 0))
                            {
                                if (regexMatch.Groups["HomeTeam"].Value.Trim().ToLower() == "bye")
                                {
                                    match.Bye = 1;
                                }
                                else
                                {
                                    match.HomeTeam = new Team();
                                    match.HomeTeam.Description = regexMatch.Groups["HomeTeam"].Value.Trim();
                                }
                            }
                            if ((regexMatch.Groups["VisitorTeam"].Success) && (regexMatch.Groups["VisitorTeam"].Value.Trim().Length > 0))
                            {
                                if (match.Bye.GetValueOrDefault(0) != 0)
                                {
                                    match.HomeTeam = new Team();
                                    match.HomeTeam.Description = regexMatch.Groups["VisitorTeam"].Value.Trim();
                                }
                                else if (regexMatch.Groups["VisitorTeam"].Value.Trim().ToLower() == "bye")
                                {
                                    match.Bye = 1;
                                }
                                else
                                {
                                    match.VisitorTeam = new Team();
                                    match.VisitorTeam.Description = regexMatch.Groups["VisitorTeam"].Value.Trim();
                                }
                            }
                            if ((!match.Bye.HasValue) || (match.Bye.Value == 0))
                            {
                                if ((regexMatch.Groups["HomeScore"].Success) && (regexMatch.Groups["HomeScore"].Value.Trim().Length > 0))
                                {
                                    match.HomeScore = int.Parse(regexMatch.Groups["HomeScore"].Value.Trim());
                                }
                                if ((regexMatch.Groups["VisitorScore"].Success) && (regexMatch.Groups["VisitorScore"].Value.Trim().Length > 0))
                                {
                                    match.VisitorScore = int.Parse(regexMatch.Groups["VisitorScore"].Value.Trim());
                                }
                            }
                            matches.Add(match);
                        }
                        UpdateMatches(matches, scrapeStep.ScrapeID, scrapeStep.Scrape.LeagueID.Value);
                        break;

                    default:
                        throw new ApplicationException("Scrape step action not implemented yet.");
                }

                // Execute any child steps
                foreach (ScrapeStep childScrapeStep in Functions.GetDatabaseDataContext()
                    .ScrapeSteps
                    .Where(ss => ss.ParentScrapeStepID == scrapeStep.ID)
                    .OrderBy(ss => ss.ExecuteOrder))
                {
                    input = ExecuteScrape(childScrapeStep.ID, input);
                }
            }

            // Return the result input
            return input;
        }

        /// <summary>
        /// Updates the matches of a given league
        /// </summary>
        public static void UpdateMatches(List<DBMatch> matches, Guid scrapeID, Guid leagueID)
        {
            using (MainDataClassesDataContext databaseDataContext = Functions.GetDatabaseDataContext())
            {
                foreach (DBMatch scrapeMatch in matches)
                {
                    // Determine the home team ID
                    Guid homeTeamID = databaseDataContext
                        .ScrapeTeamAlias
                        .Where(ta => (ta.ScrapeID == scrapeID) && (ta.TeamAlias.ToLower() == scrapeMatch.HomeTeam.Description.ToLower()))
                        .Select(ta => ta.TeamID)
                        .FirstOrDefault();

                    // Add the team if it doesn't exist
                    if (homeTeamID == Guid.Empty)
                    {
                        // Add the team
                        Team team = new Team();
                        team.ID = Guid.NewGuid();
                        team.Description = scrapeMatch.HomeTeam.Description;
                        databaseDataContext.Teams.InsertOnSubmit(team);
                        homeTeamID = team.ID;

                        // Assign the team to the league
                        LeaguesTeam leaguesTeam = new LeaguesTeam();
                        leaguesTeam.LeagueID = leagueID;
                        leaguesTeam.TeamID = team.ID;
                        databaseDataContext.LeaguesTeams.InsertOnSubmit(leaguesTeam);

                        // Add the alias to the scrape
                        ScrapeTeamAlias scrapeTeamAlias = new ScrapeTeamAlias();
                        scrapeTeamAlias.ScrapeID = scrapeID;
                        scrapeTeamAlias.TeamID = team.ID;
                        scrapeTeamAlias.TeamAlias = team.Description;
                        databaseDataContext.ScrapeTeamAlias.InsertOnSubmit(scrapeTeamAlias);

                        // Save changes to the database
                        databaseDataContext.SubmitChanges();
                    }

                    // Determine the visitor team ID
                    Guid visitorTeamID = Guid.Empty;
                    if ((scrapeMatch.VisitorTeam != null) && (scrapeMatch.VisitorTeam.Description != null))
                    {
                        visitorTeamID = databaseDataContext
                            .ScrapeTeamAlias
                            .Where(ta => (ta.ScrapeID == scrapeID) && (ta.TeamAlias.ToLower() == scrapeMatch.VisitorTeam.Description.ToLower()))
                            .Select(ta => ta.TeamID)
                            .FirstOrDefault();

                        // Add the team if it doesn't exist
                        if (visitorTeamID == Guid.Empty)
                        {
                            // Add the team
                            Team team = new Team();
                            team.ID = Guid.NewGuid();
                            team.Description = scrapeMatch.VisitorTeam.Description;
                            databaseDataContext.Teams.InsertOnSubmit(team);
                            visitorTeamID = team.ID;

                            // Assign the team to the league
                            LeaguesTeam leaguesTeam = new LeaguesTeam();
                            leaguesTeam.LeagueID = leagueID;
                            leaguesTeam.TeamID = team.ID;
                            databaseDataContext.LeaguesTeams.InsertOnSubmit(leaguesTeam);

                            // Add the alias to the scrape
                            ScrapeTeamAlias scrapeTeamAlias = new ScrapeTeamAlias();
                            scrapeTeamAlias.ScrapeID = scrapeID;
                            scrapeTeamAlias.TeamID = team.ID;
                            scrapeTeamAlias.TeamAlias = team.Description;
                            databaseDataContext.ScrapeTeamAlias.InsertOnSubmit(scrapeTeamAlias);

                            // Save changes to the database
                            databaseDataContext.SubmitChanges();
                        }
                    }

                    // Add the match
                    DBMatch databaseMatch = databaseDataContext
                        .DBMatches
                        .Where(m => (m.UTC == scrapeMatch.UTC) && (m.HomeTeamID == homeTeamID))
                        .FirstOrDefault();
                    if (databaseMatch == null)
                    {
                        databaseMatch = new DBMatch();
                        databaseMatch.ID = Guid.NewGuid();
                        databaseMatch.UTC = scrapeMatch.UTC;
                        databaseMatch.HomeTeamID = homeTeamID;
                        databaseDataContext.DBMatches.InsertOnSubmit(databaseMatch);
                    }
                    if (scrapeMatch.Round.HasValue)
                    {
                        databaseMatch.Round = scrapeMatch.Round.Value;
                    }
                    else
                    {
                        databaseMatch.Round = null;
                    }
                    databaseMatch.HomeTeamID = homeTeamID;
                    if (visitorTeamID != Guid.Empty)
                    {
                        databaseMatch.VisitorTeamID = visitorTeamID;
                    }
                    else
                    {
                        databaseMatch.VisitorTeamID = null;
                    }
                    if (leagueID != Guid.Empty)
                    {
                        databaseMatch.LeagueID = leagueID;
                    }
                    else
                    {
                        databaseMatch.LeagueID = null;
                    }
                    if (scrapeMatch.HomeScore.HasValue)
                    {
                        databaseMatch.HomeScore = scrapeMatch.HomeScore.Value;
                    }
                    else
                    {
                        databaseMatch.HomeScore = null;
                    }
                    if (scrapeMatch.VisitorScore.HasValue)
                    {
                        databaseMatch.VisitorScore = scrapeMatch.VisitorScore.Value;
                    }
                    else
                    {
                        databaseMatch.VisitorScore = null;
                    }
                    if (scrapeMatch.Bye.GetValueOrDefault(0) != 0)
                    {
                        databaseMatch.Bye = 1;
                    }
                    else
                    {
                        databaseMatch.Bye = 0;
                    }
                    databaseDataContext.SubmitChanges();
                }
            }
        }

        /// <summary>
        /// Get the HTML of the given URL
        /// </summary>
        public static string GetHtmlFromURL(string url, string post)
        {
            string html;
            HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(url);
            Encoding encode = System.Text.Encoding.GetEncoding("utf-8");
            using (StreamReader streamReader = new StreamReader(webRequest.GetResponse().GetResponseStream(), encode))
            {
                // Get the initial response
                html = streamReader.ReadToEnd();

                // Append the post data
                if ((post != null) && (post.Trim().Length > 0))
                {
                    string postData = String.Format("__VIEWSTATE={0}&{1}", ExtractViewState(html), post);
                    webRequest = (HttpWebRequest)WebRequest.Create(url);
                    webRequest.Method = "POST";
                    webRequest.ContentType = "application/x-www-form-urlencoded";
                    using (StreamWriter requestWriter = new StreamWriter(webRequest.GetRequestStream()))
                    {
                        requestWriter.Write(postData);
                        requestWriter.Close();
                    }
                    using (StreamReader responseReader2 = new StreamReader(webRequest.GetResponse().GetResponseStream()))
                    {
                        html = responseReader2.ReadToEnd();
                        responseReader2.Close();
                    }
                }
                streamReader.Close();
            }
            return html;
        }

        public static string ExtractViewState(string s)
        {
            string viewStateNameDelimiter = "__VIEWSTATE";
            string valueDelimiter = "value=\"";
            int viewStateNamePosition = s.IndexOf(viewStateNameDelimiter);
            int viewStateValuePosition = s.IndexOf(valueDelimiter, viewStateNamePosition);
            int viewStateStartPosition = viewStateValuePosition + valueDelimiter.Length;
            int viewStateEndPosition = s.IndexOf("\"", viewStateStartPosition);
            return System.Web.HttpUtility.UrlEncodeUnicode(s.Substring(viewStateStartPosition, viewStateEndPosition - viewStateStartPosition));
        }

        /// <summary>
        /// Removes lines and most tags from html.
        /// </summary>
        public static string CleanHTMLForRegex(string html)
        {
            string newHtml = html;
            newHtml = newHtml.Replace(char.ConvertFromUtf32(65533).ToString(), "") // Replace unknown blank char with a question mark
                .Replace("\r", "")
                .Replace("\n", "")
                .Replace("=", "");
            if (newHtml.IndexOf("<body") > 0)
            {
                newHtml = newHtml.Remove(0, newHtml.IndexOf("<body", StringComparison.OrdinalIgnoreCase));
                newHtml = newHtml.Remove(newHtml.IndexOf("</body", StringComparison.OrdinalIgnoreCase) + "</body".Length) + ">";
            }
            newHtml = newHtml.Replace("\t", "");
            while (newHtml.Contains("  "))
            {
                newHtml = newHtml.Replace("  ", " ");
            }
            newHtml = newHtml.Replace("&nbsp;", "");
            newHtml = RemoveTagFromHTML(newHtml, "body");
            newHtml = Regex.Replace(newHtml, "<!--[^>]*-->", "", RegexOptions.Singleline | RegexOptions.IgnoreCase);
            newHtml = RemoveTagFromHTML(newHtml, "script");
            newHtml = RemoveTagFromHTML(newHtml, "font");
            newHtml = RemoveTagFromHTML(newHtml, "p");
            newHtml = RemoveTagFromHTML(newHtml, @"[^\s><]+:[^\s><]+");
            newHtml = RemoveTagFromHTML(newHtml, "span");
            newHtml = RemoveTagFromHTML(newHtml, "br");
            newHtml = RemoveTagFromHTML(newHtml, "img");
            newHtml = RemoveTagFromHTML(newHtml, "a");
            newHtml = RemoveTagFromHTML(newHtml, "b");
            newHtml = RemoveTagFromHTML(newHtml, "strong");
            newHtml = RemoveTagFromHTML(newHtml, "u");
            newHtml = RemoveTagFromHTML(newHtml, "i");
            newHtml = Regex.Replace(newHtml, @"(?<=<\w+)\s[^>]+(?=>)", "", RegexOptions.Singleline | RegexOptions.IgnoreCase);
            newHtml = Regex.Replace(newHtml, @"(?<=<\w+>)\s+", "", RegexOptions.Singleline | RegexOptions.IgnoreCase);
            newHtml = Regex.Replace(newHtml, @"(?<=<\w+>.*)\s+(?=<)", "", RegexOptions.Singleline | RegexOptions.IgnoreCase);
            while (newHtml.Contains("  "))
            {
                newHtml = newHtml.Replace("  ", " ");
            }
            while (newHtml.Contains("> <"))
            {
                newHtml = newHtml.Replace("> <", "><");
            }
            return newHtml;
        }

        /// <summary>
        /// Removes both the start and end of a given html tag
        /// </summary>
        private static string RemoveTagFromHTML(string html, string tag)
        {
            return Regex.Replace(html, "<" + tag + "[^><]*>|</" + tag + "[^><]*>", "", RegexOptions.Singleline | RegexOptions.IgnoreCase);
        }
    }
}
